from bs4 import BeautifulSoup as bs
import re, pandas as pd, requests as rq

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                         +"AppleWebKit/537.36 (KHTML, like Gecko) "
                         +"Chrome/91.0.4472.77 Safari/537.36"}
class Area:
    placesId = ["beijing1", "shanghai2", "changsha148", "sanya61", "chongqing158",
                "hongkong38", "chengdu104", "haerbin151",
          "xian7", "guangzhou152", "hangzhou14"]
    indexId = {"北京":0, "上海":1, "长沙":2, "三亚":3, "重庆":4, "香港":5,
               "成都":6, "哈尔滨":7, "西安":8, "广州":9, "杭州":10}
    page = [str(i) for i in range(1, 3)]
    address, name, rank, level =[], [], [], []
    def __init__(self):
        self.DF = pd.DataFrame()

    def get(self, placeName): # -> DataFrame
        placeId = self.placesId[self.indexId.get(placeName)]
        # 暂时爬取2页
        for i in range(50):
            URL = self.getURL(placeId, str(i))
            print('Runing... Target:'+ URL)
            data = bs(rq.get(URL, headers = HEADERS).text, 'html.parser')   # get获取html文件 并转化为 BeautifulSoup对象
            data = str(data.contents)       # 取Html文本
            R = re.findall('<dt>([\\s\\S]*)</dt>', data)[0].replace(" ","")    # 取<dt>标签 并 去掉所有的空格
            if self.getAreaImformation(R, placeId) == True:
                print('Area: ' + placeName + ' Page:[ ' + str(i) + ' ]Done.')
        self.DF = pd.DataFrame(
            {
                '景区星级': self.level,
                '景区排名': self.rank,
                '旅游景区名称': self.name,
                '所在地': self.address
            }
        )
        return self.DF

    def getURL(self, PlaceId, pageNum):
        return 'https://you.ctrip.com/sight/' + PlaceId + '/s0-p' + pageNum + '.html'
    def getAreaImformation(self, text, placeId) -> bool:
        ADDRESS = re.findall('<ddclass="ellipsis">([^<]*)</dd>', text)
        NAME = re.findall('<ahref="/sight/' + placeId + "/[0-9]*.html[^>]*>([^<]*)</a>", text)
        RANK = re.findall('<s>([^<]*)</s>', text)
        RANK += re.findall('<sclass="g_background">([^<]*)</s>', text)
        LEVEL = re.findall('<dd>([^<]*)</dd>', text)
        MIN = min([len(ADDRESS), len(NAME), len(RANK), len(LEVEL)])
        # 字符串处理
        ADDRESS = [x.replace('\r|\n', ' ') for x in ADDRESS[:MIN]]
        NAME = [x.replace('\r|\n', ' ') for x in NAME[:MIN]]
        RANK = [x.replace('\r|\n', ' ') for x in RANK[:MIN]]
        LEVEL = [x.replace('\r|\n', ' ') for x in LEVEL[:MIN]]
        # 更新结果
        self.updateAreaImformation(ADDRESS, NAME, RANK, LEVEL)
        return True

    def updateAreaImformation(self, ADDRESS, NAME, RANK, LEVEL) -> bool:
        self.address += ADDRESS
        self.name += NAME
        self.rank += RANK
        self.level += LEVEL
        return True
    def exportAreaExcel(self, fileName) -> bool:
        writer = pd.ExcelWriter('result.xlsx')
        self.DF.to_excel(writer)
        writer.save()
        return True




# 设置列对齐
# pd.set_option('display.unicode.ambiguous_as_wide', True)
# pd.set_option('display.unicode.east_asian_width', True)

# path = 'result.xlsx'  # 文件路径
# if os.path.exists(path):  # 如果文件存在
#     # 删除文件，可使用以下两种方法。
#     os.remove(path)
A = Area()
A.get('北京')
A.exportAreaExcel('result2.xlsx')